$$f(x) = \sin(2 x) \Rightarrow f'(x) = 2 \cos(2 x)$$
plot_function("f'(x) = 2*cos(2x)", lambda x: 2 * math.cos(2 * x))
$$f_x'(x, y) = \cos(x) \ \ \ f_y'(x, y) = -2 \sin(y)$$
plot_function3D("f'_x(x, y) = cos(x)", lambda x, y: math.cos(x))
$$f(x) = x^2 + 1$$
def f(x):
return x * x + 1.0
plot_function("f(x)", f)
$$f'(x) = 2x$$
def d_f(x):
return 2 * x
def tangent_line(slope, x, y):
def line(x_):
return slope * (x_ - x) + y
return line
plot_function("f(x) vs f'(2)", f, fn2=tangent_line(d_f(2), 2, f(2)))
Approximate derivatative
$$f'(x) \approx \frac{f(x + \epsilon) - f(x-\epsilon)}{2\epsilon}$$

$$f(x) = ...$$ $$f'(x) = ...$$
def derivative(f: Callable[[float], float]) -> Callable[[float], float]:
def f_prime(x: float) -> float:
...
return f_prime
draw_boxes([("", ""), "", "", ""], [1, 2, 1], lr=True)
backprop(6)
$${\cal L}(w, b) = - \log \sigma(x;w, b)$$ where
$$m(x; w, b) = x_1 \times w_1 + x_2 \times w_2 + b$$
$${\cal L}'_w(w, b) \ \ {\cal L}'_b(w, b)$$
$$f(x) = \text{ReLU}(x)$$
draw_boxes(["$x_1$", "$f(x_1)$"], [1])
draw_boxes(["$x_2$", "$f(x_2)$"], [1])
$$f(x, y) = x \times y$$
draw_boxes([("$x$", "$y$"), "$f(x, y)$"], [1])
$$f(x) = \text{ReLU}(x)$$ $$g(x) = \log(x) $$
draw_boxes(["$x$", "$g(x)$", "$f(g(x))$"], [1, 1])
forward and backward methodsapply which handles wrapping / unwrapping$$f(x) = x \times 5$$
draw_boxes(["$x_1$", "$f(x_1)$"], [1])
class TimesFive(ScalarFunction):
@staticmethod
def forward(ctx: Context, x: float) -> float:
return x * 5
draw_boxes([("$x$", "$y$"), "$f(x, y)$"], [1])
class Mul(ScalarFunction):
@staticmethod
def forward(ctx: Context, x: float, y: float) -> float:
return x * y
x_1 = Scalar(10.0)
x_2 = Scalar(0.0)
x = Scalar(10.0)
z = TimesFive.apply(x)
def apply(cls, val: Scalar) -> Scalar:
...
unwrapped = val.data
new = cls.forward(unwapped)
return Scalar(new)
...
draw_boxes(["$x$", "$g(x)$", "$f(g(x))$"], [1, 1])
x = Scalar(10.0)
y = Scalar(5.0)
z = TimesFive.apply(x)
out = TimesFive.apply(z)
out2 = x * y
def __mul__(self, b: Scalar) -> Scalar:
return Mul.apply(self, b)
sub can be implemented with other calls.draw_boxes(["", "", ""], [1, 1], lr=False)
class TimesFive(ScalarFunction):
@staticmethod
def forward(ctx, x: float) -> float:
return x * 5
@staticmethod
def backward(ctx, d: float) -> float:
f_prime = 5
return f_prime * d
draw_boxes([("", ""), ""], [1], lr=False)
class AddTimes2(ScalarFunction):
@staticmethod
def forward(ctx, x: float, y: float) -> float:
return x + 2 * y
@staticmethod
def backward(ctx, d) -> Tuple[float, float]:
return d, 2 * d
forward is given to backwardConsider a function Square
def backward(ctx, d_out):
...
Arguments to backward must be saved in context. ::
class Square(ScalarFunction):
@staticmethod
def forward(ctx: Context, x: float) -> float:
ctx.save_for_backward(x)
return x * x
@staticmethod
def backward(ctx: Context, d_out: float) -> Tuple[float, float]:
x = ctx.saved_values
f_prime = 2 * x
return f_prime * d_out
Run Square
x = minitorch.Scalar(10)
x_2 = Square.apply(x)
x_2.history
ScalarHistory(last_fn=<class '__main__.Square'>, ctx=Context(no_grad=False, saved_values=(10.0,)), inputs=[Scalar(10.000000)])